/*
 * Copyright (c) 2009, ConnectCode Pte Ltd
 *                     http://www.webextractor360.com
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of ConnectCode nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY ConnectCode Pte Ltd ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL ConnectCode Pte Ltd BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * 
 */

using System;
using System.Collections;
using System.Text.RegularExpressions;
using System.IO;
using System.Net;
using System.ComponentModel;

using WebExtractor360.look4me.Framework;
using WebExtractor360.look4me.Model;

namespace WebExtractor360.look4me.Engine
{

   public class WebProcessingEngine
   {
       private Framework.Action m_action;
       private Uri m_initialUri;
       public Uri InitialUri
       {
           get
           {
               return m_initialUri;
           }
           set
           {
               m_initialUri = value;
           }
       }

       private Uri m_authorityUri;
       public Uri AuthorityUri
       {
           get
           {
               return m_authorityUri;
           }
           set
           {
               m_authorityUri = value;
           }
       }

       private int m_numberOfURLProcessed;
       private int NumberOfURLProcessed
       {
           get
           {
               return m_numberOfURLProcessed;
           }
           set
           {
               m_numberOfURLProcessed = value;
           }
       }

       private int m_maxURL;
       public int MaxURL
       {
           get
           {
               return m_maxURL;
           }
           set
           {
               m_maxURL = value;
           }
       }

       private Hashtable m_cache;
       public  Hashtable Cache
       {
          get
          {
              return m_cache;
          }
          set
          {
              m_cache = value;
          }
       }

       private Queue m_processqueue;
       private Queue ProcessQueue
       {
          get
          {
              return m_processqueue;
          }
          set
          {
              m_processqueue = value;
          }
       }

       private int m_externalExitIndicator;
       protected DoWorkEventArgs m_workerevent;
       protected BackgroundWorker m_worker;

       public WebProcessingEngine(Uri initialUri, int maxURL, Framework.Action action, BackgroundWorker worker, DoWorkEventArgs workerevent)
       {
           m_workerevent = workerevent;
           m_worker = worker;
           m_externalExitIndicator = 0;
           m_action = action;

           InitialUri = initialUri;
           try
           {
               AuthorityUri = new Uri(InitialUri.GetLeftPart(UriPartial.Authority));
               Console.Out.WriteLine(" {0} ", AuthorityUri.ToString());
           }
           catch (Exception)
           {
               AuthorityUri = null;
           }

           MaxURL = maxURL;
           m_processqueue = new Queue();
           m_cache = new Hashtable();
       
       }

       public void Execute( )
       {
         NumberOfURLProcessed = 0;
         AddWebPage( InitialUri, InitialUri.AbsoluteUri );
         try
         {
            while (m_externalExitIndicator==0 && ProcessQueue.Count>0 && NumberOfURLProcessed<MaxURL)
            {          
               Source source = null;
               source = (Source)m_processqueue.Dequeue();
               Process(source);
               ExtractorCurrentResultsValueObject vo = new ExtractorCurrentResultsValueObject();
               vo.Links = "Page Traversed - "+source.Uri.AbsoluteUri.ToString();
               m_action.CommonCallBack(vo);
               NumberOfURLProcessed++;

               if (m_worker.CancellationPending)
               {
                   m_workerevent.Cancel = true;
                   m_externalExitIndicator = 1;
               }
            }
         }
         catch (Exception ex)
         {
            Console.WriteLine(ex.ToString());
         }
      }

      public void HandleLinks( Source source )
      {
            int   counter  = 0;

            Match m = new Regex(@"(?:href\s*=)(?:[\s""']*)(?!#|mailto|location.|javascript)(?<PARAM1>.*?)(?:[\s>""'])", RegexOptions.IgnoreCase).Match(source.Data);
            while( m.Success )
            {
                if (Singleton.GetInstance().ShareClientModelHolder.ShareSearchOptionsVO.ReportLinksFound)
                {
                    ExtractorCurrentResultsValueObject vo = new ExtractorCurrentResultsValueObject();
                    vo.NoProgress = 1;
                    vo.Links = "Links Found - " + m.Groups["PARAM1"].ToString();
                    Console.WriteLine("MyLinks-" + vo.Links);
                    m_action.CommonCallBack(vo);
                }

               if (AddWebPage(source.Uri, m.Groups["PARAM1"].ToString()))
               {                 
                   counter++;
               }
               m = m.NextMatch( );
            }
      }

      private bool AddWebPage( Uri baseUri, string newUri )
      {
         string urlWithoutBookmarks;
         if (newUri == null || newUri.Trim().Length <= 0)
             urlWithoutBookmarks = null;
         else
         {
             if (newUri.IndexOf("#", 0) == -1)
                 urlWithoutBookmarks = newUri;
             else
                 urlWithoutBookmarks = newUri.Substring(0, newUri.IndexOf("#", 0));
         }

         Uri uri = null;

         try
         {
             uri = new Uri(baseUri, urlWithoutBookmarks);
         }
         catch (Exception)
         {
             return false;
         }

         if ( ! ValidPage( uri.LocalPath ) || m_cache.Contains( uri ) )
            return false;

         Source source = new Source( uri );
         if (Singleton.GetInstance().ShareClientModelHolder.ShareSearchOptionsVO.MadCrawl == true
             ||
             AuthorityUri == null
             ||
             uri.AbsoluteUri.StartsWith(AuthorityUri.AbsoluteUri))
         {
             m_processqueue.Enqueue(source);
             m_cache.Add(uri, source);
             return true;
         }
         else
             return false;
      }
      
      public static string[] m_validPageExtensions = 
      {
          "html", 
          "shtml", 
          "htm", 
          "php", 
          "asp", 
          "jsp", 
          "php3", 
          "aspx", 
          "pl", 
          "py", 
          "cfm",
          "ttvn",
          "do"
      };

      private bool ValidPage( string pageURL ) 
      {
         if (Singleton.GetInstance().ShareClientModelHolder.ShareSearchOptionsVO.IgnoreCriterion)
              return true;

         if (pageURL.IndexOf(".", 0) == -1 || 
             pageURL[pageURL.Length - 1] == '/') 
            return true;

         string htmlextension=null;

         if (pageURL == null)
             htmlextension = null;
         else
         {
             if (pageURL.IndexOf(".",0) + 1 > pageURL.Length)
                 htmlextension = "";
             else
                 htmlextension = pageURL.Substring(pageURL.IndexOf(".", 0) + 1);
         }
         
         for (int x = 0; x < m_validPageExtensions.Length; x++)
         {
             if (htmlextension.Equals(m_validPageExtensions[x]))
                 return true;
         }
         return false;

      }

      public bool Process(Source source)
      {
          bool success = false;
          try
          {
              WebResponse response = null;
              WebRequest request = WebRequest.Create(source.Uri);       
              try
              {
                  response = request.GetResponse();

                  if ((response is HttpWebResponse && ((HttpWebResponse)response).StatusCode.ToString()=="OK") ||
                      response is FileWebResponse)
                  {
                      StreamReader sr = new StreamReader(response.GetResponseStream());
                      source.Data = sr.ReadToEnd();
                      HandleLinks(source);
                      doHandleContents(source);    
                  }
                  success = true;
              }
              catch (Exception ex)
              {
                  Console.WriteLine(ex);
                  success = false;
              }
              finally
              {
                  if (response != null)
                      response.Close();
              }
          }
          catch (Exception ex)
          {
              Console.WriteLine(ex.ToString());
              success= false;
          }

          return success;
      }

      virtual public void doHandleContents(Source source)
      {

      }
   
   }
}
